package com.html;

import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import javax.xml.bind.JAXBException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.DomSerializer;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.json.JSONObject;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.DOMException;
import org.w3c.dom.NodeList;

import cn.edu.hfut.dmic.webcollector.net.HttpRequesterJsonImpl;

import com.model.xmlElement.DataColumn;
import com.model.xmlElement.DataColumnInput;
import com.model.xmlElement.DataColumnOutput;
import com.model.xmlElement.DataColumnParse;
import com.model.xmlElement.DataColumns;
import com.model.xmlElement.PageDefault;
import com.model.xmlElement.output.OutputFormater;
import com.tool.JaxbReadXmlTool;

/**
 * *********************************************
 * @author Administrator
 * @FileName ParseHtml.java
 * @Description 解析HTML类
 **********************************************
 */
public class ParseHtml {
	public static final Logger LOG = LoggerFactory.getLogger(ParseHtml.class);
	
	private String seed;
	private Document document;
	private org.w3c.dom.Document dom;
	private TagNode tagNode;
	private DataColumns dataColumns;
	private List<PageDefault> pageDefaults;
	private HtmlCleaner htmlCleaner;
	private DomSerializer domSerializer;
	private XPath xPath;
	private Map<String,JSONObject> jsonObjectMap = new HashMap<String,JSONObject>();
	
	public ParseHtml(String seed,Document document,DataColumns dataColumns) {
		super();
		this.seed = seed;
		this.document = document;
		this.dataColumns = dataColumns;
		this.htmlCleaner = new HtmlCleaner();
		this.domSerializer = new DomSerializer(new CleanerProperties());
		this.xPath = XPathFactory.newInstance().newXPath();
		this.tagNode = htmlCleaner.clean(document.html());
		try {
			this.dom = domSerializer.createDOM(tagNode);
		} catch (ParserConfigurationException e) {
			e.printStackTrace();
		}
		initPageDefaultValue();
	}
	
	/**
	 * 初始化页面默认值
	 */
	private void initPageDefaultValue(){
		pageDefaults = dataColumns.getPageDefaults();
		if(dom != null && pageDefaults != null && pageDefaults.size() > 0){
			for(PageDefault pd : pageDefaults){
				if(pd.getKey() != null && !"".equals(pd.getKey()) && pd.getXpath() != null && !"".equals(pd.getXpath())){
					Object result = null;
					try {
						result = xPath.evaluate(pd.getXpath(), dom, XPathConstants.NODESET);
					}  catch (XPathExpressionException e) {
						e.printStackTrace();
					}
					StringBuilder value = null;
					if (result instanceof NodeList) {
						value = new StringBuilder();
						NodeList nodeList = (NodeList) result;
						int resultLength = nodeList.getLength();
						for (int i = 0; i < resultLength; i++) {
							value.append(nodeList.item(i).getNodeValue());
						}
						pd.setValue(value.toString());
					}
				}
				LOG.info("set ${" + pd.getKey() + "} to " + pd.getValue());
			}
		}
	}
	
	/**
	 * 解析HTML
	 */
	public DataColumns parseHtml(){
		if(dataColumns != null && dataColumns.size() > 0){
			//检测当前种子返回的Document是否符合xml文件中定义的dataColumns规则
			if(!documentCheck(document,dataColumns)){
				return null;
			}
			
			/**
			 * 保存DataColumn和Element或者Node[]的对应关系
			 * 当需要获取的值来自其他元素的时候使用
			 * 处理两种元素org.jsoup.nodes.Element、org.w3c.dom.Node[]
			 */
			DataColumnOutput dataColumnOutput;
			String dataoutput_default_value;
			Object object;
			for(DataColumn dataColumn : dataColumns){
				dataColumnOutput = dataColumn.getDataColumnOutput();
				if(dataColumnOutput != null){
					dataoutput_default_value = dataColumnOutput.getDefaultValue();
					if(dataoutput_default_value != null){
						dataColumn.getDataColumnOutput().setValue(new StringBuilder(dataoutput_default_value));
						continue;
					}
				}
				object = getElementObjectByDataColumn(document, dataColumn);
				if(object == null){
					break;
				}
				if(dataColumn.getDataColumnUrl() != null){
					dataColumn.getDataColumnOutput().setValue(parseJson(dataColumn, object));
				}else{
					dataColumn.getDataColumnOutput().setValue(parseContent(dataColumn, object));
				}
			}
		}
		return dataColumns;
	}
	
	/**
	 * 解析返回的数据
	 * @param dataColumn
	 * @param object
	 * @return
	 */
	public StringBuilder parseJson(DataColumn dataColumn,Object object){
		StringBuilder stringBuilder = new StringBuilder();
		if(object instanceof JSONObject){//解析JSONObject
			JSONObject jsonObject = (JSONObject) object;
			String returnType = dataColumn.getDataColumnUrl().getReturnType();
			stringBuilder.append(new ParseJsoupElement2(seed,dataColumn, returnType, jsonObject).parse());
		}
		return stringBuilder;
	}
	
	/**
	 * 解析返回的数据
	 * @param dataColumn
	 * @param object
	 * @return
	 */
	public StringBuilder parseContent(DataColumn dataColumn,Object object){
		//解析数据
		StringBuilder stringBuilder = new StringBuilder();
		DataColumnOutput dataColumnOutput = dataColumn.getDataColumnOutput();
		if(dataColumnOutput != null){
			String outputType = dataColumn.getDataColumnOutput().getOutputType();
			if(object instanceof org.jsoup.nodes.Element){//解析org.jsoup.nodes.Element
				Element element = (org.jsoup.nodes.Element) object;
				stringBuilder.append(new ParseJsoupElement(seed,dataColumn, outputType, element).parse());
			}else if(object instanceof org.w3c.dom.Node[]){//解析org.w3c.dom.Node[]
				org.w3c.dom.Node[] nodes = (org.w3c.dom.Node[])object;
				if(dataColumn.getDataColumnPattern() != null){//存在pattern
					stringBuilder.append(new ParseW3CNodeArrayWithPattern(this, dataColumn, nodes).parse());
				}else{
					stringBuilder.append(new ParseW3CNodeArrayNoPattern(this, dataColumn, nodes).parse());
				}
			}
			String dataColumnValue = stringBuilder.toString();
			stringBuilder.delete(0, stringBuilder.length());
			//获取匹配数据
			if(dataColumnValue != null && !"".equals(dataColumnValue)){
				OutputFormater outputFormater = dataColumn.getDataColumnOutput().getOutputFormater();
				if(outputFormater != null){
					FormaterDataOutput formaterDataOutput = new FormaterDataOutput(outputFormater, dataColumnValue);
					formaterDataOutput.formaterData();
					dataColumnValue = formaterDataOutput.getDataColumnValue();
				}
			}
			stringBuilder.append(dataColumnValue);
		}
		return stringBuilder;
	}
	
	/**
	 * 根据XML和返回的Document定位元素
	 * @param document
	 * @param dataColumn
	 * @return
	 */
	public Object getElementObjectByDataColumn(Document document,DataColumn dataColumn){
		Object element = null;
		DataColumnInput dataColumnInput = dataColumn.getDataColumnInput();
		if(dataColumnInput != null){
			if(dataColumnInput.getValueReferer() != null && !dataColumnInput.getValueReferer().equals("")){//引用
				if(dataColumnInput.getValueReferer().startsWith("http") || dataColumnInput.getValueReferer().startsWith("https")){
					//URL引用
					String refererUrl = dataColumnInput.getValueReferer();
					//赋值URL中的引用组成完整URL
					for(PageDefault pd : pageDefaults){
						if(pd.getKey() != null && !"".equals(pd.getKey()) && pd.getValue() != null && !"".equals(pd.getValue())){
							refererUrl = refererUrl.replace("${" + pd.getKey() + "}", pd.getValue());
						}
					}
					LOG.info("refererUrl-->" + refererUrl);
					
					//判断当前URL是否已经抓取过,存在的话在已经抓取过的Map中获取
					boolean needfetch = true;
					Iterator<Entry<String,JSONObject>> itea = jsonObjectMap.entrySet().iterator();
					Entry<String,JSONObject> entry;
					while(itea.hasNext()){
						entry = itea.next();
						if(entry.getKey().equals(refererUrl)){
							needfetch = false;
							element = entry.getValue();
							break;
						}
					}
					//单签URL还没有抓取过,抓取
					if(needfetch){
						HttpRequesterJsonImpl requester = new HttpRequesterJsonImpl();
						try {
							element = requester.getResponse(refererUrl);
							jsonObjectMap.put(refererUrl, (JSONObject)element);
						} catch (Exception e) {
							e.printStackTrace();
						}
					}
				}else{
					//常规引用
					for(DataColumn column : dataColumns){
						if(column.getDataColumnInput() != null && column.getDataColumnInput().getName().equals(dataColumnInput.getValueReferer())){
							dataColumn = column;
							break;
						}
					}
				}
			}
		}
		DataColumnParse dataColumnParse = dataColumn.getDataColumnParse();
		if(dataColumnParse != null){
			if(dataColumnParse.getId() != null && !dataColumnParse.getId().equals("")){//id
				element = document.getElementById(dataColumnParse.getId());
			}else{
				if(dataColumnParse.getClazz() != null && !dataColumnParse.getClazz().equals("")){//class
					Element classElement = document.getElementsByAttributeValue("class", dataColumnParse.getClazz()).get(Integer.valueOf(dataColumnParse.getClazzIndex()));
					int childIndex = (dataColumnParse.getChildIndex() == null || dataColumnParse.getChildIndex().equals("")) ? -1 : Integer.valueOf(dataColumnParse.getChildIndex());
					if(childIndex == -1){
						element = classElement;
					}else{
						element = classElement.child(childIndex);
					}
				}else if(dataColumnParse.getXpath() != null && !dataColumnParse.getXpath().equals("")){//Xpath
					if(dom == null){
						return null;
					}
					Object result = null;
					try {
						result = xPath.evaluate(dataColumnParse.getXpath(), dom, XPathConstants.NODESET);
					}  catch (XPathExpressionException e) {
						e.printStackTrace();
					}
					if (result instanceof NodeList) {
						NodeList nodeList = (NodeList) result;
						int resultLength = nodeList.getLength();
						org.w3c.dom.Node[] nodes = new org.w3c.dom.Node[resultLength];
						for (int i = 0; i < resultLength; i++) {
							nodes[i] = nodeList.item(i);
						}
						element = nodes;
					}
				}
			}
		}
		return element;
	}
	
	/**
	 * 
	 * @param document
	 * @param DataColumns
	 * @return 网页对比XML配置
	 * 注释：主要检测3个	id，class，xpath
	 */
	public boolean documentCheck(Document document,DataColumns dataColumns){
		boolean flag = true;
		if(document == null){
			flag = false;
			return flag;
		}
		
		DataColumnOutput dataColumnOutput;
		DataColumnParse dataColumnParse;
		String dataoutput_default_value;
		String element_id;
		String element_class;
		String element_classIndex;
		String element_childIndex;
		String element_xpath;
		for(DataColumn dataColumn : dataColumns){
			//检测是否有默认值
			dataColumnOutput = dataColumn.getDataColumnOutput();
			if(dataColumnOutput != null){
				dataoutput_default_value = dataColumnOutput.getDefaultValue();
				if(dataoutput_default_value != null){
					continue;
				}
				if(!dataColumnOutput.isOutputNotNull()){
					continue;
				}
			}
			dataColumnParse = dataColumn.getDataColumnParse();
			if(dataColumnParse != null){
				//检测element_id
				element_id = dataColumnParse.getId();
				if(element_id != null && !element_id.equals("")){
					if(document.getElementById(element_id) == null){
						flag = false;
						break;
					}
				}
				//检测elemenet_class
				element_class = dataColumnParse.getClazz();
				element_classIndex = dataColumnParse.getClazzIndex();
				element_childIndex = dataColumnParse.getChildIndex();
				if(element_class != null && !element_class.equals("")){
					Elements elements = document.getElementsByAttributeValue("class", element_class);
					if(elements == null || elements.size() == 0 || elements.size() - 1 < Integer.valueOf(element_classIndex)){
						flag = false;
						break;
					}else{
						Elements children = elements.get(Integer.valueOf(element_classIndex)).children();
						int children_size = children.size();
						if(!element_childIndex.equals("") && children_size - 1 < Integer.valueOf(element_childIndex)){
							flag = false;
							break;
						}
					}
				}
				//检测element_xpath
				element_xpath = dataColumnParse.getXpath();
				if(element_xpath != null && !element_xpath.equals("")){
					Object result = null;
					try {
						result = xPath.evaluate(dataColumnParse.getXpath(), dom, XPathConstants.NODESET);
					}  catch (XPathExpressionException e) {
						flag = false;
						break;
					} catch(DOMException e){
						flag = false;
						break;
					}
					if (result == null || (result != null && result instanceof NodeList && ((NodeList) result).getLength() == 0)) {
						flag = false;
						break;
					}
				}
			}
		}
		return flag;
	}
	
	/**
	 * xml文件转javaBean
	 * @param fileName
	 * @return
	 * @throws JAXBException 
	 * @throws IOException 
	 */
	public DataColumns Xml2JavaBean(String fileName) throws JAXBException{
		return JaxbReadXmlTool.readString(DataColumns.class, fileName);
	}

	public String getSeed() {
		return seed;
	}

	public void setSeed(String seed) {
		this.seed = seed;
	}

	public Document getDocument() {
		return document;
	}

	public void setDocument(Document document) {
		this.document = document;
	}

	public org.w3c.dom.Document getDom() {
		return dom;
	}

	public void setDom(org.w3c.dom.Document dom) {
		this.dom = dom;
	}

	public TagNode getTagNode() {
		return tagNode;
	}

	public void setTagNode(TagNode tagNode) {
		this.tagNode = tagNode;
	}

	public DataColumns getDataColumns() {
		return dataColumns;
	}

	public void setDataColumns(DataColumns dataColumns) {
		this.dataColumns = dataColumns;
	}

	public List<PageDefault> getPageDefaults() {
		return pageDefaults;
	}

	public void setPageDefaults(List<PageDefault> pageDefaults) {
		this.pageDefaults = pageDefaults;
	}

	public HtmlCleaner getHtmlCleaner() {
		return htmlCleaner;
	}

	public void setHtmlCleaner(HtmlCleaner htmlCleaner) {
		this.htmlCleaner = htmlCleaner;
	}

	public DomSerializer getDomSerializer() {
		return domSerializer;
	}

	public void setDomSerializer(DomSerializer domSerializer) {
		this.domSerializer = domSerializer;
	}

	public XPath getxPath() {
		return xPath;
	}

	public void setxPath(XPath xPath) {
		this.xPath = xPath;
	}

	public Map<String, JSONObject> getJsonObjectMap() {
		return jsonObjectMap;
	}

	public void setJsonObjectMap(Map<String, JSONObject> jsonObjectMap) {
		this.jsonObjectMap = jsonObjectMap;
	}
	
	

}
